{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\" style=\"margin-top: 1em;\"><ul class=\"toc-item\"></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected cross entropy loss if the model:\n",
      "- learns neither dependency: 0.661563238158\n",
      "- learns first dependency: 0.519166699707\n",
      "- learns both dependency: 0.454454367449\n"
     ]
    }
   ],
   "source": [
    "print(\"Expected cross entropy loss if the model:\")\n",
    "print(\"- learns neither dependency:\", -(0.625 * np.log(0.625) + 0.375 * np.log(0.375)))\n",
    "print(\"- learns first dependency:\", \n",
    "      -0.5 * (0.875 * np.log(0.875) + 0.125 * np.log(0.125))\n",
    "      -0.5 * (0.625 * np.log(0.625) + 0.375 * np.log(0.375)))\n",
    "print(\"- learns both dependency:\", \n",
    "      -0.5 * (0.75 * np.log(0.75) + 0.25 * np.log(0.25))\n",
    "      -0.25 * (2 * 0.5 * np.log(0.5) - 0.25 * (0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES']=''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_steps = 5\n",
    "batch_size = 200\n",
    "num_classes = 2\n",
    "state_size = 16\n",
    "learning_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_data(size = 1000000):\n",
    "    X = np.array(np.random.choice(2, size=(size,)))\n",
    "    Y = []\n",
    "    for i in range(size):\n",
    "        threshold = 0.5\n",
    "        if X[i-3] == 1:\n",
    "            threshold += 0.5\n",
    "        if X[i-8] == 1:\n",
    "            threshold -= 0.25\n",
    "        if np.random.rand() > threshold:\n",
    "            Y.append(0)\n",
    "        else:\n",
    "            Y.append(1)\n",
    "    return X, np.array(Y)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_batch(raw_date, batch_size, num_steps):\n",
    "    raw_x, raw_y = raw_date\n",
    "    data_length = len(raw_x)\n",
    "    \n",
    "    #partition raw data into batches and stak them vertically in a data matrix\n",
    "    batch_partition_length = data_length // batch_size\n",
    "    data_x = np.zeros([batch_size, batch_partition_length], dtype=np.int32)\n",
    "    data_y = np.zeros([batch_size, batch_partition_length], dtype=np.int32)\n",
    "    # do partition \n",
    "    for i in range(batch_size):\n",
    "        data_x[i] = raw_x[batch_partition_length * i : batch_partition_length * (i + 1)]\n",
    "        data_y[i] = raw_y[batch_partition_length * i : batch_partition_length * (i + 1)]\n",
    "    # do epoch\n",
    "    epoch_size = batch_partition_length // num_steps\n",
    "    \n",
    "    for i in range(epoch_size):\n",
    "        x = data_x[:, i * num_steps:(i + 1) * num_steps]\n",
    "        y = data_y[:, i * num_steps:(i + 1) * num_steps]\n",
    "        yield(x, y)\n",
    "        \n",
    "def gen_epochs(n, num_steps):\n",
    "    for i in range(n):\n",
    "        yield gen_batch(gen_data(), batch_size, num_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "x = tf.placeholder(tf.int32, [batch_size, num_steps], name='input_placeholder')\n",
    "y = tf.placeholder(tf.int32, [batch_size, num_steps], name='labels_placeholder')\n",
    "init_state = tf.zeros([batch_size, state_size])\n",
    "\n",
    "# x_one_hot = tf.one_hot(x, num_classes)\n",
    "# rnn_inputs = tf.unstack(x_one_hot, axis = 1)\n",
    "\n",
    "# 优化二：dynamic方式\n",
    "rnn_inputs = tf.one_hot(x, num_classes)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"rnn/transpose:0\", shape=(200, 5, 16), dtype=float32)\n",
      "Tensor(\"rnn/while/Exit_2:0\", shape=(200, 16), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "cell = tf.nn.rnn_cell.BasicRNNCell(state_size)\n",
    "rnn_outputs, final_state = tf.nn.dynamic_rnn(cell, rnn_inputs, initial_state=init_state)\n",
    "print(rnn_outputs)\n",
    "print(final_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.variable_scope('softmax'):\n",
    "    W = tf.get_variable('W', [state_size, num_classes])\n",
    "    b = tf.get_variable('b', [num_classes], initializer=tf.constant_initializer(0.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# logits = [tf.matmul(rnn_output, W) + b for rnn_output in rnn_outputs]\n",
    "# predictions = [tf.nn.softmax(logit) for logit in logits]\n",
    "\n",
    "# y_as_list = tf.unstack(y, num=num_steps, axis=1)\n",
    "\n",
    "# losses = [tf.nn.sparse_softmax_cross_entropy_with_logits(labels=label, logits=logit) for \n",
    "#          logit, label in zip(logits, y_as_list)]\n",
    "\n",
    "# 优化二：dynamic方式\n",
    "# dynamic优化 logits predictions losses计算方式做相应修改\n",
    "logits = tf.reshape(\n",
    "            tf.matmul(tf.reshape(rnn_outputs, [-1, state_size]), W) + b,\n",
    "            [batch_size, num_steps, num_classes])\n",
    "predictions = tf.nn.softmax(logits)\n",
    "\n",
    "losses = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_loss = tf.reduce_mean(losses)\n",
    "train_step = tf.train.AdamOptimizer(learning_rate).minimize(total_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "EPOCH 0\n",
      "Average loss at step 100 for last 100 steps: 0.562411904633\n",
      "Average loss at step 200 for last 100 steps: 0.486812432706\n",
      "Average loss at step 300 for last 100 steps: 0.486584925056\n",
      "Average loss at step 400 for last 100 steps: 0.484679380357\n",
      "Average loss at step 500 for last 100 steps: 0.475878683925\n",
      "Average loss at step 600 for last 100 steps: 0.478745310605\n",
      "Average loss at step 700 for last 100 steps: 0.480136155188\n",
      "Average loss at step 800 for last 100 steps: 0.479659578502\n",
      "Average loss at step 900 for last 100 steps: 0.476197741032\n",
      "\n",
      "EPOCH 1\n",
      "Average loss at step 100 for last 100 steps: 0.486250113845\n",
      "Average loss at step 200 for last 100 steps: 0.477325642109\n",
      "Average loss at step 300 for last 100 steps: 0.473715016246\n",
      "Average loss at step 400 for last 100 steps: 0.473608194292\n",
      "Average loss at step 500 for last 100 steps: 0.475305302441\n",
      "Average loss at step 600 for last 100 steps: 0.472920046449\n",
      "Average loss at step 700 for last 100 steps: 0.474952809513\n",
      "Average loss at step 800 for last 100 steps: 0.474010813236\n",
      "Average loss at step 900 for last 100 steps: 0.473171933591\n",
      "\n",
      "EPOCH 2\n",
      "Average loss at step 100 for last 100 steps: 0.481668917537\n",
      "Average loss at step 200 for last 100 steps: 0.472274899185\n",
      "Average loss at step 300 for last 100 steps: 0.471134241223\n",
      "Average loss at step 400 for last 100 steps: 0.473415424228\n",
      "Average loss at step 500 for last 100 steps: 0.46909432441\n",
      "Average loss at step 600 for last 100 steps: 0.470332282782\n",
      "Average loss at step 700 for last 100 steps: 0.468834472299\n",
      "Average loss at step 800 for last 100 steps: 0.469336364567\n",
      "Average loss at step 900 for last 100 steps: 0.467326209247\n",
      "\n",
      "EPOCH 3\n",
      "Average loss at step 100 for last 100 steps: 0.478766950071\n",
      "Average loss at step 200 for last 100 steps: 0.467629900873\n",
      "Average loss at step 300 for last 100 steps: 0.469408822656\n",
      "Average loss at step 400 for last 100 steps: 0.468912951648\n",
      "Average loss at step 500 for last 100 steps: 0.469557642937\n",
      "Average loss at step 600 for last 100 steps: 0.470407552123\n",
      "Average loss at step 700 for last 100 steps: 0.467511906326\n",
      "Average loss at step 800 for last 100 steps: 0.47064059943\n",
      "Average loss at step 900 for last 100 steps: 0.469684744775\n",
      "\n",
      "EPOCH 4\n",
      "Average loss at step 100 for last 100 steps: 0.479618255198\n",
      "Average loss at step 200 for last 100 steps: 0.465925890207\n",
      "Average loss at step 300 for last 100 steps: 0.467453899682\n",
      "Average loss at step 400 for last 100 steps: 0.46753465414\n",
      "Average loss at step 500 for last 100 steps: 0.467402789891\n",
      "Average loss at step 600 for last 100 steps: 0.465057616234\n",
      "Average loss at step 700 for last 100 steps: 0.465190475583\n",
      "Average loss at step 800 for last 100 steps: 0.464708399177\n",
      "Average loss at step 900 for last 100 steps: 0.467943740189\n",
      "\n",
      "EPOCH 5\n",
      "Average loss at step 100 for last 100 steps: 0.476640506387\n",
      "Average loss at step 200 for last 100 steps: 0.465876362324\n",
      "Average loss at step 300 for last 100 steps: 0.46668150723\n",
      "Average loss at step 400 for last 100 steps: 0.468191360831\n",
      "Average loss at step 500 for last 100 steps: 0.465463904142\n",
      "Average loss at step 600 for last 100 steps: 0.465428844392\n",
      "Average loss at step 700 for last 100 steps: 0.466021747589\n",
      "Average loss at step 800 for last 100 steps: 0.465170513988\n",
      "Average loss at step 900 for last 100 steps: 0.465069873929\n",
      "\n",
      "EPOCH 6\n",
      "Average loss at step 100 for last 100 steps: 0.473194414079\n",
      "Average loss at step 200 for last 100 steps: 0.466776555777\n",
      "Average loss at step 300 for last 100 steps: 0.466533381939\n",
      "Average loss at step 400 for last 100 steps: 0.466559114754\n",
      "Average loss at step 500 for last 100 steps: 0.466519871056\n",
      "Average loss at step 600 for last 100 steps: 0.463720062077\n",
      "Average loss at step 700 for last 100 steps: 0.463591589034\n",
      "Average loss at step 800 for last 100 steps: 0.463742262721\n",
      "Average loss at step 900 for last 100 steps: 0.462973934114\n",
      "\n",
      "EPOCH 7\n",
      "Average loss at step 100 for last 100 steps: 0.473489004076\n",
      "Average loss at step 200 for last 100 steps: 0.462803415358\n",
      "Average loss at step 300 for last 100 steps: 0.462954886556\n",
      "Average loss at step 400 for last 100 steps: 0.463726673722\n",
      "Average loss at step 500 for last 100 steps: 0.463273652494\n",
      "Average loss at step 600 for last 100 steps: 0.462380851209\n",
      "Average loss at step 700 for last 100 steps: 0.463210323751\n",
      "Average loss at step 800 for last 100 steps: 0.460975980163\n",
      "Average loss at step 900 for last 100 steps: 0.460662111938\n",
      "\n",
      "EPOCH 8\n",
      "Average loss at step 100 for last 100 steps: 0.468888665736\n",
      "Average loss at step 200 for last 100 steps: 0.460451487005\n",
      "Average loss at step 300 for last 100 steps: 0.461186262369\n",
      "Average loss at step 400 for last 100 steps: 0.462319329977\n",
      "Average loss at step 500 for last 100 steps: 0.461069799662\n",
      "Average loss at step 600 for last 100 steps: 0.459338513911\n",
      "Average loss at step 700 for last 100 steps: 0.459995259643\n",
      "Average loss at step 800 for last 100 steps: 0.461678331792\n",
      "Average loss at step 900 for last 100 steps: 0.460330373645\n",
      "\n",
      "EPOCH 9\n",
      "Average loss at step 100 for last 100 steps: 0.469377160966\n",
      "Average loss at step 200 for last 100 steps: 0.459423698485\n",
      "Average loss at step 300 for last 100 steps: 0.459846995771\n",
      "Average loss at step 400 for last 100 steps: 0.461201184094\n",
      "Average loss at step 500 for last 100 steps: 0.461094816923\n",
      "Average loss at step 600 for last 100 steps: 0.458385087848\n",
      "Average loss at step 700 for last 100 steps: 0.458247444928\n",
      "Average loss at step 800 for last 100 steps: 0.460637509823\n",
      "Average loss at step 900 for last 100 steps: 0.460597479343\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f52f07e8128>]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xl8nGW5//HPNTOZtNmbNknbNGm673spUHaQTbRVFimiUgRRkAPnKEfhuMuPw1EUgQMqqCjrAQXUApVSWSuU0nRf0iUttEmbNGn2NHvm+v0xM2mWSTJtkil95nq/XrzMPPPMzD3j9Js71708oqoYY4yJDq4T3QBjjDGRY6FvjDFRxELfGGOiiIW+McZEEQt9Y4yJIhb6xhgTRSz0jTEmiljoG2NMFLHQN8aYKOI50Q3obNiwYZqTk3Oim2GMMSeVdevWHVbVtN7O+8SFfk5ODrm5uSe6GcYYc1IRkX3hnGflHWOMiSIW+sYYE0Us9I0xJopY6BtjTBSx0DfGmChioW+MMVHEQt8YY6KIY0L/SGML96/cxcaCyhPdFGOM+cRyTOg3tvh46I3dbLLQN8aYboUV+iJyiYjsFJF8EbkzxP1LRaRURDYG/rux3X3ZIvK6iOSJyHYRyem/5h/l9fjfSnOrbyCe3hhjHKHXbRhExA08AlwIFAJrRWSZqm7vdOrzqnpriKd4ErhHVVeKSAIwIKkc4xYAmiz0jTGmW+H09BcA+aq6V1WbgOeAxeE8uYhMBTyquhJAVWtVte64W9uDGFegp9+iA/H0xhjjCOGEfiZQ0O52YeBYZ1eIyGYReUFEsgLHJgKVIvKSiGwQkfsCfzn0O5dL8LjEyjvGGNOD/hrIfRnIUdWZwErgicBxD3AWcAdwCjAWWNr5wSJyk4jkikhuaWnpcTcixu2y0DfGmB6EE/oHgKx2t0cFjrVR1TJVbQzc/D0wL/BzIbAxUBpqAf4GzO38Aqr6mKrOV9X5aWm9bgfdrRi3WE3fGGN6EE7orwUmiMgYEfECS4Bl7U8QkRHtbi4C8to9NkVEgkl+PtB5ALjfeD0umlos9I0xpju9zt5R1RYRuRVYAbiBx1V1m4j8FMhV1WXAbSKyCGgBygmUcFS1VUTuAN4QEQHWAb8bmLdi5R1jjOlNWFfOUtXlwPJOx37Y7ue7gLu6eexKYGYf2hg2f+jb7B1jjOmOY1bkgtX0jTGmN44Kfa/HTbPV9I0xplvOCn23zdM3xpieOCr0raZvjDE9c1zoW03fGGO656zQ99iUTWOM6YmjQt/rFlucZYwxPXBU6NviLGOM6ZkDQ98Gco0xpjuOC30r7xhjTPccFfpeG8g1xpgeOSv0bXGWMcb0yFGhbzV9Y4zpmbNC32OLs4wxpifOCv3AlE1V6+0bY0wojgp9r1tQhRafhb4xxoTiqNCPcfvfjg3mGmNMaM4M/Rbr6RtjTCjOCn2P/+3YYK4xxoTmqND3ugWw8o4xxnTHWaHvsZq+Mcb0xFGhbwO5xhjTM0eGfpMN5BpjTEiOCn2v9fSNMaZHjgp9K+8YY0zPHBb6/tk7tqe+McaE5qzQt3n6xhjTI0eF/tGavg3kGmNMKI4KfavpG2NMz8IKfRG5RER2iki+iNwZ4v6lIlIqIhsD/93Y6f4kESkUkYf7q+GhxNiKXGOM6ZGntxNExA08AlwIFAJrRWSZqm7vdOrzqnprN09zN/Bun1oahuCKXBvINcaY0MLp6S8A8lV1r6o2Ac8Bi8N9ARGZB2QArx9fE8NnNX1jjOlZOKGfCRS0u10YONbZFSKyWUReEJEsABFxAb8E7uhzS8NgNX1jjOlZfw3kvgzkqOpMYCXwROD4LcByVS3s6cEicpOI5IpIbmlp6XE3IsY2XDPGmB71WtMHDgBZ7W6PChxro6pl7W7+Hvh54OfTgbNE5BYgAfCKSK2q3tnp8Y8BjwHMnz//uGszwYHcRqvpG2NMSOGE/lpggoiMwR/2S4Avtj9BREaoalHg5iIgD0BVr213zlJgfufA708xLuvpG2NMT3oNfVVtEZFbgRWAG3hcVbeJyE+BXFVdBtwmIouAFqAcWDqAbe6WyyV4XGKhb4wx3Qinp4+qLgeWdzr2w3Y/3wXc1ctz/An40zG38BjFuF02e8cYY7rhqBW54K/r2zx9Y4wJzXGh7/W4rLxjjDHdcF7ouy30jTGmO44L/RiP1fSNMaY7zgt9t8v20zfGmG44M/RtINcYY0JyXOh73TZP3xhjuuO40I+xgVxjjOmWM0O/xQZyjTEmFOeFvscGco0xpjuOC32r6RtjTPccF/pW0zfGmO45LvS9tjjLGGO65bjQt3n6xhjTPWeGvpV3jDEmJMeFvg3kGmNM9xwX+v55+hb6xhgTivNC3wZyjTGmW84L/UBNX9WC3xhjOnNc6HvdAkCLz0LfGGM6c1zox7j9b8kGc40xpivHhb7XEwh923TNGGO6cFzoB3v6NlffGGO6clzoey30jTGmW44L/RiPfyDX5uobY0xXzgt9G8g1xphuOTb0rbxjjDFdOS70vW09fZu9Y4wxnTku9K28Y4wx3Qsr9EXkEhHZKSL5InJniPuXikipiGwM/Hdj4PhsEVktIttEZLOIXN3fb6CzGLcN5BpjTHc8vZ0gIm7gEeBCoBBYKyLLVHV7p1OfV9VbOx2rA76iqrtFZCSwTkRWqGplfzQ+lBiP1fSNMaY74fT0FwD5qrpXVZuA54DF4Ty5qu5S1d2Bnw8CJUDa8TY2HFbTN8aY7oUT+plAQbvbhYFjnV0RKOG8ICJZne8UkQWAF9hzXC0NU9s2DNbTN8aYLvprIPdlIEdVZwIrgSfa3ykiI4CngOtVtUsai8hNIpIrIrmlpaV9akjblE2r6RtjTBfhhP4BoH3PfVTgWBtVLVPVxsDN3wPzgveJSBLwKvA9Vf0g1Auo6mOqOl9V56el9a36ExzItZq+McZ0FU7orwUmiMgYEfECS4Bl7U8I9OSDFgF5geNe4K/Ak6r6Qv80uWdem7JpjDHd6nX2jqq2iMitwArADTyuqttE5KdArqouA24TkUVAC1AOLA08/AvA2cBQEQkeW6qqG/v3bRzVNk/fyjvGGNNFr6EPoKrLgeWdjv2w3c93AXeFeNzTwNN9bOMxifHY7B1jjOmOA1fkWk3fGGO647zQd1lN3xhjuuO40He5BI9LLPSNMSYEx4U++BdoWU3fGGO6cmTox7hdtjjLGGNCcG7oW3nHGGO6cGToe91i8/SNMSYER4Z+jMdlA7nGGBOCM0PfbQO5xhgTimND32r6xhjTlSND3+u2efrGGBOKI0PfX96x0DfGmM6cG/otVtM3xpjOHBn6Xo+LRuvpG2NMF44MfX9P30LfGGM6c2Toez02kGuMMaE4MvRtINcYY0JzcOjbQK4xxnTm2NC3xVnGGNOVI0PfFmcZY0xojgx9m71jjDGhOTP07cpZxhgTkjNDP1DTV7XgN8aY9hwZ+rEe/9uy3r4xxnTkyNCPcQuADeYaY0wnDg39YE/fQt8YY9pzdOjbXH1jjOnIkaHvdVtN3xhjQnFk6Md4AjV9m6tvjDEdhBX6InKJiOwUkXwRuTPE/UtFpFRENgb+u7HdfdeJyO7Af9f1Z+O7YzV9Y4wJzdPbCSLiBh4BLgQKgbUiskxVt3c69XlVvbXTY1OBHwHzAQXWBR5b0S+t74bV9I0xJrRwevoLgHxV3auqTcBzwOIwn/9iYKWqlgeCfiVwyfE1NXzBmn6TlXeMMaaDcEI/Eyhod7swcKyzK0Rks4i8ICJZx/jYfuW1xVnGGBNSfw3kvgzkqOpM/L35J47lwSJyk4jkikhuaWlpnxtjNX1jjAktnNA/AGS1uz0qcKyNqpapamPg5u+BeeE+NvD4x1R1vqrOT0tLC7ft3QquyLWavjHGdBRO6K8FJojIGBHxAkuAZe1PEJER7W4uAvICP68ALhKRISIyBLgocGxAtfX0raZvjDEd9Dp7R1VbRORW/GHtBh5X1W0i8lMgV1WXAbeJyCKgBSgHlgYeWy4id+P/xQHwU1UtH4D30YHV9I0xJrReQx9AVZcDyzsd+2G7n+8C7urmsY8Dj/ehjcfMavrGGBOaM1fkWk3fGGNCcmToe62nb4wxITky9G0g1xhjQnNm6HtsGwZjjAnFkaFvWysbY0xojgz9toFcK+8YY0wHjgx9ESHGLTaQa4wxnTgy9ME/mGuhb4wxHTk89K2mb4wx7Tk69G32jjHGdOTY0Pe6xebpG2NMJ44N/RiP1fSNMaYz54a+lXeMMaYLZ4d+iw3kGmNMe44Nfa+Vd4wxpgvnhr4tzjLGmC4cG/q2OMsYY7pydOg32eIsY4zpwNGhb/P0jTGmo7CukXsyGhTjIq+4mjN/9iajhgxmXFoC/3HhRIYlxJ7ophljzAnj2J7+bRdM4Jvnjmf+6CE0typ/WVfIl//wIVV1zf3+WjuKqzn3vrcormro9+c2xpj+5Nie/sSMRO64eFLb7VW7S7nhT7ks/dOHPHXDqSTE9t9bX7XrMB+X1bF672E+P2dUvz2vMcb0N8f29Ds7a0IaD10zh82FVXztiVwamlv77bnziqsB2FJY3W/PaYwxAyFqQh/gkunD+cVVM1m9t4x7l+f12/PuKKoBYMuByn57TmOMGQhRFfoAn58zik9NSWfV7sP98nwtrT7yS2oRga0Hqmn12TRRY8wnV9SFPsCsUSnsPXyE6oa+D+p+dPgITa0+zhw/jPrmVvaW1vZDC40xZmBEZejPGJUMwNYDVX1+rrxif2nnynn+AdzNhX1/TmOMGShRGfozR6UA/RPQO4qq8biEi6cNJ87rZks//CIxxpiBEpWhnxrvZdSQwWzpj9AvrmFcWgKDYtxMH5kc8dCvb2plf1ldRF/TGHPyCiv0ReQSEdkpIvkicmcP510hIioi8wO3Y0TkCRHZIiJ5InJXfzW8r2aNSmFzP8y22VFUzeQRiQBMz0xm28EqWiK40dv/vrmbTz+0iibbcsIYE4ZeQ19E3MAjwKXAVOAaEZka4rxE4HZgTbvDVwGxqjoDmAd8XURy+t7svpsxKpmC8nrKjzQd93NU1TVzsKqBycOTAJg5KpmGZh/5ERzMXftxObWNLewuqYnYaxpjTl7h9PQXAPmquldVm4DngMUhzrsb+BnQfi8CBeJFxAMMBpqAT8QKppmZ/sHcnsoxJdUNvLWzpNutG3YEFmUFe/rBAeL+KBuFo7nV19b+bQc/ER+rMeYTLpzQzwQK2t0uDBxrIyJzgSxVfbXTY18AjgBFwH7gF6pafvzN7T/TAwG9uSB0ieftnSVc9MC7XP/Htcy5+3UWP/wvfvn6TmobW9rO2RGYuTMl0NMfMzSehFhPxOr6uw7V0NDsL+tsj7LQ31RQSUUf/kozJlr1eSBXRFzA/cC3Q9y9AGgFRgJjgG+LyNgQz3GTiOSKSG5paWlfmxSWpEExjB0Wz+ZOAe3zKQ/8cxfX/2ktw5MG8buvzOffzp9AjNvFw2/l84sVO9vO3VFcQ0pcDBlJ/p07XS5h2sikiE3b3FTgf53hSYPYdjB6Zg01trRy1aOr+e27e050U4w56YQT+geArHa3RwWOBSUC04G3ReRj4DRgWWAw94vAa6rarKolwHvA/M4voKqPqep8VZ2flpZ2fO/kOMwcldyhFNPqU772ZC4P/HM3n5+TyV9vOYMLp2bwHxdO5IWbF/KFeVk8u2Y/ByrrAX95Z/LwRESkw3PmFVVH5KpdGwsqGBIXw0XTMth+sBpfhFcDP/3BPp7+YF9EXxNg96Famlp8fFR6JOKvbczJLpzQXwtMEJExIuIFlgDLgneqapWqDlPVHFXNAT4AFqlqLv6SzvkAIhKP/xfCjn5+D8dtxqgUiqsbKKn2D0M8tfpj3thRwvcvm8Ivr5rFYK+7w/m3fWoCAP/7xm58PmVncU3bIG7Q9MxkGlt87D408IO5mwqqmJWVwrSRSRxpamVfeWSnbv7xvY94/L2PIvqacLSstj/C79cYJ+g19FW1BbgVWAHkAX9W1W0i8lMRWdTLwx8BEkRkG/5fHn9U1c19bXR/mRWs6xdWUVRVz30rdnLWhGHccOaYDr33oMyUwXzx1Gz+sq6Qd3eXUtfUyuThiR3OCS78GujN12obW9hVUsOsUSlMG+l/H5Es8TS3+thXVse+srqITxfNK/KPXxSU16Fqex0ZcyzCqumr6nJVnaiq41T1nsCxH6rqshDnnhvo5aOqtap6lapOU9Wpqnpf/za/b6aOTMIlsLmwkh/9fRutqtzzuRkhAz/om+eNx+t28d0X/b+7Jo/o2NMfnRpH4iAPa/YO7Hj11gNVqMLs7BQmZCTgcUlEZ/DsK6ujxae0+pSPyyJbZgnOmjrS1NqnKbfGRKOoXJEbFOf1MDEjkWc/3M/r2w9x+wUTyR4a1+Nj0hJjuf6MHA5VNyICEzMSOtzvcgmfm53Jsk0HKawYuPLDxsCso1mjUoj1uJmQkRjR0N/Tbi1CJEpZQapKXlFN22UvCyrqI/baAEcaW7jjL5vaSoLGnGyiOvQBZmQmc7i2icnDE7nxrDFhPebrZ48jcZCH0alxxHm7XoHr5nPH4RLhkbcGbnbJpoJKslPjSI33AjBtZBLbDlRFrNyRX+IPepGjP0dCaU0j5UeauHBqBhD5uv6/8g/zwrpCVuYdiujrGtNfoj70F4xJxe0S7r18BjHu8D6O5LgYHloyh+9d1mVhMgAjUwZz9SlZvLCuoMfefmNLK5V1x1ee2FRQyayslLbb00cmUXakiUPVjcf1fMdqT2ktw5MGMWrI4IiuBg7uanpRIPQLIhz6mwv9f2FF8q8bY/pT1If+5XNH8f6d5zMne8gxPe68yeltvc1Qbj53HAC/fjt0b7+kuoFF//se5//yHQ5Wdi1RvJF3iFc2Hww5DbOkuoGDVQ3Mbhf60zIjO5i7p/QI49MTGJ+WENGefnAQd052CsMSYk9A6Ps/312HbNsLc3KK+tB3u4SMpEH9/rzB3v5fcgva5vUHFZTXcdWjqymoqKOxuZV/+78NHeb1v76tmK89mcutz27g0gdX8fq24g5lm2A9f3ZWctuxKSOSEInMdgyqyp6SWsalxTMhI5G9h49E7IphO4qqGZE8iJQ4L9mpgyNa3lHVdqFvPX1zcor60B9It5w7HoD7X9/Fgcp6qhua2X2ohqt+u5rKumaeufFU/ueKmazbV8F9gZW+6/dXcNtzG5iRmcyvrp5FU6uPm55ax+ceeY/XtxXj8ymbCivxuKRtqiZAQqyHnKHxEenpl9Q0UtvYwrhAT7+pxRexHveO4hqmBGZMZaXGRTT095XVUVXfzNi0eA7XNto2EOak1HUU0vSbkSmDWXJKNk99sI8X1xe2HR+WEMvzXz+NycOTmJM9hDUflfHYu3vJSBrEw2/uJiNpEH9YegrDEmL57MyRvLi+kP99M5+bnlrH+PQEWn3K5BGJDIrpuHhs6sgkNnWzl1BZbSPPrS1g6cIc4mP79n/7nkA5Z1xaQtsCtvySWnKGxffpeXvT2NJKfkkt509OByA7NY5XNhfR3OoLezymLzYF6vlXzhvFz1/bye6SWhaMSR3w1w3y+ZSyI02kJcZG7DWDVmwrJr+klm+eNz7ir236l/X0B9gPPzuVR788j59dMYPvXzaFOy6ayF9vWdhhJe/3L5vKtJFJ3P3KdlwiPHH9grYpiR63i6tPyebtO87lwSWziXG7+OjwEeaFGIOYNjKJwor6LruCNrf6uOWZ9dy3YifPrtnf5/cUnK45Pj2B8en+Kau7I1DX31NyhBafdujpt/qUosrITJ/cXFhFrMfFZTNGAJGv67+wvpAzfvYmh07AdNGnVu/j4TfzI77Vh+l/1tMfYDFuFxdPG97jOYNi3Pz62rn85OXt3H7BhJA9Zo/bxeLZmSyaNZINBZWMG5bQ5ZxguWdTYSVnTzy6h9H//GMHaz4qZ1hCLE+v2ccNZ47B5ep+AVpv8ktqSYj1kJ4Yi4iQkRQbkcHc4CDulMBW1llD/Gsq9pfX9bq+oj9sLqxk2sgkslPjiPe62R3h0H8v/zBNLT7e2VXKF+Zn9f6AfrTzUA31za0cqKwnK3XgP2szcKyn/wkxemg8jy89pcM0zFBEhLnZQ0iOi+ly3+xRKSQO8nDLM+t5cvXHtPqUv288wB/+9RFLF+bwg89MYV9ZHe/s7ttOpntKjzAuLb5t5fL49ATyIzBtc0dxNV6Pi5yh/l+KwaCPRF2/pdXH1gPVzByVgogwPiMx4oO56/dXAPDOrsjsRBtUfqSJ0hr/VOCdxTZr6WRnoe8gyXExvPpvZzEnO4Uf/n0bl//6Pb774mYW5KTyvcumcOn0EQxLiOWp1X3bGXNPaS3j0o/+pTEhPZE9pUcGfGHYjuIaJmUk4gnU74cnDSLGLRQM4MrnoPzSWuqbW5kVmDE1MT0housTSmsaKSivx+tx8a/dhyN6Sc72ZaxddoW2k56FvsNkD43jya8u4IGrZ1NQUU/y4BgevnYOMW4XXo+LaxZk8dbOkuOebVPb2EJRVQPj0o6G/rj0BGobWyju51rzntLaDts95BVVd9jgzu0SRg2JzAyezYFrFwQ31JuYkcjh2qaI7f2zIdDLv/bUbKrqm9kUoWs2wNHQj/O6bVGaA1joO5CI8Lk5mbz7nfNY8e9nk554dB3CF0/NxiXC02v8vX1V5aX1hXzzmfUhF4l1trf06MydoAnBwdxjDIQPPyrnhj+t5ZZn1nUZIKyqa+bK37zPp+5/h2//eRMb9ldwuLapbRA3aNSQwRGZLrqpsJLEWA9jAqWlCRnB9xyZnu/6/ZXEuIVvnDMOl0S2xLOzuIbkwTHMGz3EFqU5gIW+gyXEekiJ83Y4NiJ5MBdOyeDPaws4WFnPzU+v51t/3sTyrUUsevi9trpxd9rP3AkK/hzOYG5Lq4+V2w9x5W/e5wuPruaDvWUs31LM87kFHc574I1dVNU3s+SULF7ZfJDP//p94Oj1iIOyU+MiEvqbC6uYMSq5bQB8Yoa/HbsitBp5/f4Kpo5MJiNpELOzUiIa+rsO1TAxI4FJGYnkl9RGbCGeGRgW+lHoy6ePpqKumXN/8TZv7ijhvz49mdduP5s4r5slj33AS+sL2XWoht+v2suX/7CG6//4IUcC1wbeU3IEj0sY3W62zNB4L0PiYtqmbRaU13Hpg6s4++dv8b2/buG1rcWs21fOj5dt47R73+BrT+ZSVNXAjz87lbXf/xSnjknl3uV5bYOF+SU1PLV6H0sWZHPv5TN5+z/P5ZoFWUwensiMzOQO7yU7NY6KumaqG0JfvL4/NLa0sqO4uq20AzAieRAJsZ6I9PSbW31sLqxkbrb/9c+ZmM7mwsqIlJZU/RcLmpiRyMSMRBojuBDvRKtvauWyh1bxr92HT3RT+pVN2YxCC8cNZdaoZFp8yv1fmM2kQJ387988g5ufWce3/ryp7dxxafF8XFbHzc+s5w/XzSe/pJbsoXEdFkOJCOPTE9hTUktBeR1LHvuAmoZmTslJ5W8bDvBMYG2A1+PigsnpLJ6dyQVT0tue457Pz+DSB9/lnle388CSOdz9Sh6DvW6+feFEwP/Xyb2Xzwz5XoLTBwvK6zqsUO5PeUU1NLdq20V32r/nSJQ7dhbX0NDsa9sf6pxJafzqn7tYtbuUxbMzB/S1D1U3Ut3QwqThiW0lrZ2HagZ8IV571z3+ITMyk7nj4kkRe02ArQer2Hawmjd2HOLMCcMi+toDyUI/CokIL91yBu5Oc/WHxHt58qun8syafQyOcXP2xDRGpgzmuQ/3c+dLW/jui5vJL63tUM8PGp+ewCubirj60dUcaWrl2a+dxvTMZJpafGzYX0FxdQPnTkoneXDXqabj0xO4+ZxxPPRmPmmJsbyzq5TvXzaFoQm9rzzNjkDoB3fWnNlpOu3EjATeyCsJ+Zj6plZ+9c9dnDl+WIc1E8cjWHIL9vRnZCYzJC6Gd3YNfOgHf6lNzEhkQqCktftQTa9rT/pLTUMz7+4upaiqPuKhH7x+9o4iZ41jWOhHqc6BH+T1uLj+jI7XFViyIJtD1Y386p+7AELuLjo+PZGaxgI8buHZr53aFsBej4tTxw7ttT23nDeeZZsO8rtVHzF2WDxfOT0nrPdxtKfvH4RWVdZ+XEFRVT11Ta3UNbXiEkiJiyElzkvWkLgO4xHhWLn9EOmJsYxM7rgx38SMRP6cW0hZbWOHX1CHqhv42pO5bC6s4vm1Baz8VsfB9GO1fl8F6YmxZKYMBvz/3505IY13dx3G59M+LbTrTfvQT4j1kJkyOKLrEzYX+q8Qt7ukltrGFhL6uIXIsdh6IBD6xdWoao9X1DuZWOibsNx2wXgO1TTw7Jr9bbN12jt7wjBOyRnCTxdP7zLDJhyDYtz89+Uz+MZT6/jRoml4PWFe22BwDEmDPOwvr6O51cePl21rKyd15xdXzeLKeaPCev5Vu0tZtfsw3/v0lC7/6Nt6viW1baG/9UAVNzyxltqGFn702anc+48d/OBvW/ntl+Ydd2is31/JnOyUDo8/Z2IaL286yPaiaqZnDsxfOOAvLaUlxrZdrGdiRmRKWkHBqaqq/r+4Fo6LXJllSyD0K+qaOVTdyPDk/t+N90Sw0DdhERHuXjydM8YNa9vwrL0JGYn85RsL+/QaC8cNY/0PLmxbfBWu7KFx5BVV89U/rWXV7sN8/ZyxXDUvi/hYN3ExHlpVqaxroqKumV+s2Mn3/rqFaSOTev3l1OpT7nk1j6zUwXxl4egu909sN21zbFo8T6/ex+9WfURqvJcXbl7IlBFJNLb4+J9/7OCVzUV8dtbIY3pfAIdrG9lfXse1p2Z3OH72RH/4XfO7DxgzLJ6s1DjSEmLxqdLiU1QVlwgxbhcxbuG8yenHFZjBmTtH33Mi7+WX0dLqO+b/n47H+v2VDE8aRHF1AxsLIhf6dU0t7CmtZeG4oby/p4y84moLfRN93C7hspkjBvQ1jidIslPjWL6lGI9L+PmVM0PuSxPsqT50zRwue2gVNz+9jmX/diZJg7qOMQS9uK6QHcU1PPzFOcR63F3uH540iMRYD799Zy93v5JHs8/HRVMz+H+fm9E0LLa1AAAOuUlEQVS2E+aNZ47hH1uK+NGybSwcN7TLOMWBynpe21oMwFXzR3Vpz4b9/vGEuaM7brCXnjiIB5fMZu3H5ewvr2fbgSrKjjThcQlul+ASodWnNLf6aGj28dQH+3j93885pj2KfD5l16Faliw4+nlOyEikqdXHvvK6kGM7/UlV2bC/gk9NyWDtx+Xd7iA7ELYfrMan/h1V399TRl5RNedN6trZORlZ6JuT3rzRqazZW87DX5zL6eN6Hj9IS4zlkWvnsuSxD/jOXzbzmy/NbSubtK/bHmls4Rev72ROdkrbrpqdiQizs1NY+3E5V5+SxVfPHMOYTrNaPG4X9101i8889C9ufmY9p41JxeN20eJT3t1V2nZBHIAHVu7iy6eP5qtnjmnbZXX9/go8LukyVRVg8ezMsAZyi6rqufD+d/mvv27hqRsWhF1mKqyop765lUkZR9dGBHv9u4prBjz095XVUVHXzNzRQ2hu9bF6b9mAvl57wdLOGeOHkZky2FGDuRb65qT31TNy+OoZOWGH2Sk5qdx5yWTuWZ7Hp+5/h/qmVioC21GfMiaVs8YPo6CijpKaxg6/FEL57Zfm4VMlsYe/GCZmJPK9y6bw38vz+PCj8rbj0zOT+M4lk7h0+giONLbwm7f38Jt39vDou3tJHhxDQqyHiiNNTB2Z1OXaCcdiRPJgvnvpZH7wt638ZV1h2Dt07gwO4rbb+mJ8egIi/iuHXTrjuJsUlg0F/nr+nOwUGptb+dvGgxRV1TMiefDAvjD+0E9LjCUjaRCThyeyo3jgr0gXKRb65qR3PAOkN541huqGZrYfrCYlzr+4rKnVx/t7yrhneR4Al80YwbzRPV8kJdwL0ly3MIfrFuagqrT6lFbVLiWjR66dy97SWv624QDldU3UNrRQ29jC5+eEN+jck2sXZPPyxoPc82oe505KC2s2UXDAtv3AfZzXQ9aQuIhsvLZhfyXxXjcT0hOpb2oFYFNBZURCf+uBqra/rqaMSOLtXaU0NLf26ZfvJ4WFvolKIsK3Lwo977u4qoHcfeWcMQCDhiKCxy3d/sMbm5bAt7ppV1+4XMK9V8zg0gdX8YO/beW7l0wmJc5L8uAYmlt9lB1poqy2EY/LxZQRiYgIO4tryEwZ3OWvmIkZCd2uRG5q8fHR4SOMTYvv89XMNuyvZFZWCm6XMHVkEl63iw0FlVwyfWDHleqaWsgvqW17nckjEmn1KfkltQM6UypSLPSN6WR48iA+M/PYZ9p80o1LS+D2CyZw34qdrNh2qNvzpo1M4rqFOWwvqu4wcydoQkYi7+wq7XCZypLAdN5n1uyntKaROK+beaOHcNrYoSyaNbLLhVdU/YPE49MTQq4ZqW9qJa+omq+fMxaAWI+bKSOT2Lh/4Adz84r8g7jte/rg39rbQt8Yc1K55dxxzM0ewqHqBirqmqisa8brcTEswUtqfCyHqht4avU+vvPCZgAumNJ1xsrEjASaW5W7XtpCU4uP0ppGcveV09yqnDcpjYunDSevqJoP9pZz34qd3L9yF5fPyeTW88eTmTKYV7cU8eg7e9leVM01C7JCbrGx5UAVLT5lTtbRWUuzRyXzl3WFtPq028WF/SG4EjcY+jlD44n1uNqu3Hays9A3JoqISK8znK49NZvVe8t4ZXMRV87tOp4wf3QqcV43K7YWk5rgZWi8ly+flsOXTx/dZfZSUVU9j76zl//7cD8vbTjA0HgvJTWNjE2L59MzhvN/HxYwbWQyXzqt4zqI4KKs2dlHt76YnZ3CE6v3sbukpsM1poPe2VXK/rIjXLMgu09rCLYcqGZYQiwZSf4ZVG6XMMlBg7kW+saYDkSEheOGdbsQKis1jm0/uTisAfQRyYP58aJp3HLuOB59dy8fHfaH8gWT01GgrmktP162jYkZiSwYc3TQfMP+SrJT49qmrgLMDvT6N+6v7BD6JdUN/OTl7by6pQiAlzYc4P4vzO7yC6g7Ww9UkZ4U2za47R/ETerw/qYMT2Jl3iFHbMcQ1q9DEblERHaKSL6I3NnDeVeIiIrI/HbHZorIahHZJiJbRMQZy9qMiWLHGnzpSYP4wWem8vjSU7hwagauwCKyB5fMISs1jlueWdd2ER9VZf3+CuZkd9zgLmdoHMmDY9gU2ACvvqmVJ97/mAvuf4eVeYe446KJPHD1bPaU1PLpB1fxzJp9vV7C889rC1j08L+4+Ffv8taOEuqbWtldUtNlXcTkEYkdrhXc2eo9ZTz1wT7+vvEAb+441HaxoU+iXnv6IuIGHgEuBAqBtSKyTFW3dzovEbgdWNPumAd4Gviyqm4SkaHAwG18bow5qSQPjuF3X5nH5x55nwvvf4dJwxMZMyyBkppG5mZ3XIUsIszKSmH1njJ+vGwbL64vpKahhdPHDuW/L5/R1rM/bexQ/vOFTXzvr1vZfrCauxdPD7kp3e/e3cs9y/M4c/wwyo40cf2f1nLxtAx8SpcB2+BfFnnFNaQnHe23NrX4+NlrO/jDvz7q1FZ49sbTeiylqSqrdh+moq6JGLcLj0tIjfcyP6fnacJ9FU55ZwGQr6p7AUTkOWAxsL3TeXcDPwP+s92xi4DNqroJQFUjt6TOGHNSGJ+eyNM3nspL6wvZWVzDGzsO4XYJC0ME5pysFN7dVcoza/ZxyfQRXHtqNqeOSe3wl8fw5EE8cf0CfrZiB4++s5eGZh8/v3Jm2+CvqvLL13fx8Fv5XDZjBL+6ejY+Ve5+ZXvbZn0zRnUM/SmBK7blFVVzTmCr7MKKOm59dgMbCypZujCHb5wzjiNNLVTXN/Mfz2/kOy9u4rXbz+52LceTq/fxo2XbOhybnZXC3755xnF+kuEJJ/QzgfbXsisETm1/gojMBbJU9VURaR/6EwEVkRVAGvCcqv68j202xjjM7KwUZgeuV6CqNLb4Qi6EWrowh5Epg7hgSkaHen9nLpdw5yWTiYvx8Kt/7qKhpZV/v2ACL286yF83HqCgvJ6r52fx35fPaPtlcM/nZ3DG+GHkFVUzPKljFTolzsuI5EG8v6eMOK+btR9X8PYO/7UUfn3tXD7daauOn185i6sfW83PX9vBTxZP79K+zYWV/L9Xt3PepDS+/5mptLT690mKDXN32b7o80CuiLiA+4Gl3Tz/mcApQB3whoisU9U3Oj3HTcBNANnZ2V2exBgTPUSk25WvQ+K9XH1KeBkhItz+qQkMinFx7z928OrmIkTgjHHD+PaFk1g8e2SXsYlPzxjRJcCDpo1M4p95Jby7q5ThSYM4f0o637pwIqOHdh0wXjAmlaULc/jjex9z6YwRnNbumhJV9c1889n1pCXEcv8XZjMk3tvl8QNJehvoEJHTgR+r6sWB23cBqOq9gdvJwB4gOHIxHCgHFgHjgUtV9brAuT8AGlT1vu5eb/78+Zqbm9uX92SMMR38feMBDlU38NlZI497G4ePDx9h84Eq5mankJkyuNfB7LqmFi59cBWq/kuRpsT5VzZ/4+l1vJFXwvNfP515nXZP7YtAh3p+r+eFEfoeYBdwAXAAWAt8UVW3dXP+28AdqporIkOAN/D39puA14Bfqeqr3b2ehb4xxinW7C3j6sc+APxXkUsaFMPh2ka+f9kUbjxrbL++Vrih32t5R1VbRORWYAXgBh5X1W0i8lMgV1WX9fDYChG5H/8vCgWW9xT4xhjjJKeOHcqzXzuVzYVVVBxpoqKuidFD47nhzDG9P3iA9NrTjzTr6RtjzLELt6c/8EPFxhhjPjEs9I0xJopY6BtjTBSx0DfGmChioW+MMVHEQt8YY6KIhb4xxkQRC31jjIkin7jFWSJSCuzrw1MMAw73U3OcwD6Pjuzz6Mo+k45O1s9jtKqm9XbSJy70+0pEcsNZlRYt7PPoyD6Pruwz6cjpn4eVd4wxJopY6BtjTBRxYug/dqIb8Aljn0dH9nl0ZZ9JR47+PBxX0zfGGNM9J/b0jTHGdMMxoS8il4jIThHJF5E7T3R7TgQRyRKRt0Rku4hsE5HbA8dTRWSliOwO/G//XaPtJCAibhHZICKvBG6PEZE1ge/K8yIS2YuUnkAikiIiL4jIDhHJE5HT7fsh/xH497JVRP5PRAY5+TviiNAXETfwCHApMBW4RkSmnthWnRAtwLdVdSpwGvDNwOdwJ/CGqk7Af/nKaPuleDuQ1+72z/BftnM8UAHccEJadWI8CLymqpOBWfg/l6j9fohIJnAbMF9Vp+O/OuASHPwdcUToAwuAfFXdq6pNwHPA4hPcpohT1SJVXR/4uQb/P+hM/J/FE4HTngA+d2JaGHkiMgq4DPh94LYA5wMvBE6Jms9DRJKBs4E/AKhqk6pWEsXfjwAPMDhwPfA4oAgHf0ecEvqZQEG724WBY1FLRHKAOcAaIENViwJ3FQMZJ6hZJ8IDwHcAX+D2UKBSVVsCt6PpuzIGKAX+GCh3/V5E4oni74eqHgB+AezHH/ZVwDoc/B1xSuibdkQkAXgR+HdVrW5/n/qna0XFlC0R+QxQoqrrTnRbPiE8wFzgN6o6BzhCp1JONH0/AALjF4vx/0IcCcQDl5zQRg0wp4T+ASCr3e1RgWNRR0Ri8Af+M6r6UuDwIREZEbh/BFByotoXYWcAi0TkY/wlv/Px17RTAn/KQ3R9VwqBQlVdE7j9Av5fAtH6/QD4FPCRqpaqajPwEv7vjWO/I04J/bXAhMCIuxf/QMyyE9ymiAvUq/8A5Knq/e3uWgZcF/j5OuDvkW7biaCqd6nqKFXNwf+deFNVrwXeAq4MnBZNn0cxUCAikwKHLgC2E6Xfj4D9wGkiEhf49xP8TBz7HXHM4iwR+TT++q0beFxV7znBTYo4ETkTWAVs4WgN+7/w1/X/DGTj38H0C6pafkIaeYKIyLnAHar6GREZi7/nnwpsAL6kqo0nsn2RIiKz8Q9qe4G9wPX4O39R+/0QkZ8AV+Of/bYBuBF/Dd+R3xHHhL4xxpjeOaW8Y4wxJgwW+sYYE0Us9I0xJopY6BtjTBSx0DfGmChioW+MMVHEQt8YY6KIhb4xxkSR/w8tFV4nD/X01QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f52f1822c88>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def train_network(num_epochs, num_steps, state_size=4, verbose=True):\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        training_losses = []\n",
    "        for idx, epoch in enumerate(gen_epochs(num_epochs, num_steps)):\n",
    "            training_loss = 0\n",
    "            training_state = np.zeros((batch_size, state_size))\n",
    "            if verbose:\n",
    "                print(\"\\nEPOCH\", idx)\n",
    "            for step, (X, Y) in enumerate(epoch):\n",
    "                tr_losses, training_loss_, training_state, _= \\\n",
    "                    sess.run([losses, \n",
    "                              total_loss,\n",
    "                              final_state,\n",
    "                              train_step],\n",
    "                            feed_dict={x:X, y:Y, init_state:training_state})\n",
    "                training_loss += training_loss_\n",
    "                if step % 100 == 0 and step > 0:\n",
    "                    if verbose:\n",
    "                        print(\"Average loss at step\", step,\n",
    "                              \"for last 100 steps:\", training_loss/100)\n",
    "                    training_losses.append(training_loss/100)\n",
    "                    training_loss = 0\n",
    "    return training_losses\n",
    "training_losses = train_network(10, num_steps, state_size=state_size)\n",
    "plt.plot(training_losses)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16.0,
    "lenType": 16.0,
    "lenVar": 40.0
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
